/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package TextNor.Expand;

import LM.LanguageModel;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.StringTokenizer;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.stream.XMLStreamWriter;

/**
 *
 * @author Pham_Thanh
 */
public final class ExpandHasPw {

    public ArrayList expnAr = new ArrayList();
    public ArrayList gFullAr = new ArrayList();
    public HashMap<String, ArrayList> expnHm = new HashMap<String, ArrayList>();
    public Pef peff = new Pef();
    public LanguageModel lm = new LanguageModel();
//    public static void main(String argv[]) throws XMLStreamException, FileNotFoundException, UnsupportedEncodingException, IOException{
//        ExpandHasPw ehpw=new ExpandHasPw("textNor\\trainset_punc_nsw_gr_split_group_num_splitCom_asignCom_greek_alphaTree_other.xml","textNor\\trainset_punc_nsw_gr_split_group_num_splitCom_asignCom_greek_alphaTree_other_expnTest.xml");
//    }

    public ExpandHasPw(String inf, String ouf) throws XMLStreamException, FileNotFoundException, UnsupportedEncodingException, IOException {

        System.out.println("package TextNor.Expand; class ExpandHasPw");

        readDataFile();
        readXMLFile(inf);

        guessFull();
        writeToXMLFile(ouf);
    }

    public void readXMLFile(String inf) throws XMLStreamException, FileNotFoundException, IOException {
//        FileInputStream fis = new FileInputStream(inf);
//        XMLStreamReader xmlStreamReader = XMLInputFactory.newInstance().createXMLStreamReader(fis);
//
//        while (xmlStreamReader.hasNext()) {
//            int eventCode = xmlStreamReader.next();
//            switch (eventCode) {
//                case XMLStreamConstants.START_ELEMENT:
//                    String localName = xmlStreamReader.getLocalName();
//                    if (localName.equals("W")) {
//                        String atValue = xmlStreamReader.getAttributeValue(0);
//                        if(atValue.equals("EXPN")){
//                            String expn=xmlStreamReader.getElementText();
//                            ArrayList flist=expnHm.get(expn);
//                            if(flist==null){
//                                System.out.println("Tu khong co la: ");
//                                System.out.println(expn);
//                                
//                            }else{
//                                expnAr.add(expn);
//                            }
//                            
//                        }
//                    }
//            }
//        }
//        fis.close();
        FileInputStream fis = new FileInputStream(inf);
        XMLStreamReader xmlStreamReader = XMLInputFactory.newInstance().createXMLStreamReader(fis);

        FileOutputStream fos = new FileOutputStream("textNor\\preExpn.xml");
        OutputStreamWriter osw = new OutputStreamWriter(fos, "UTF8");
        XMLStreamWriter xmlStreamWriter = XMLOutputFactory.newInstance().createXMLStreamWriter(osw);

        xmlStreamWriter.writeStartElement("root");

        int index = 0;
        ArrayList expnDoc = null;
        while (xmlStreamReader.hasNext()) {
            int eventCode = xmlStreamReader.next();
            switch (eventCode) {

                case XMLStreamConstants.START_ELEMENT:
                    String localName = xmlStreamReader.getLocalName();
                    if (localName.equals("record")) {
                        String RecId = xmlStreamReader.getAttributeValue(0);
                        xmlStreamWriter.writeStartElement(localName);
                        xmlStreamWriter.writeAttribute("RecId", RecId);
                        expnDoc = new ArrayList();


                    }

                    if (localName.equals("URL")) {
                        String url = xmlStreamReader.getElementText();

                        xmlStreamWriter.writeStartElement(localName);
                        xmlStreamWriter.writeCharacters(url);
                        xmlStreamWriter.writeEndElement();

                    }
                    if (localName.equals("Infomation")) {
                        xmlStreamWriter.writeStartElement(localName);


                    }

                    if (localName.equals("W")) {
                        String atVal = xmlStreamReader.getAttributeValue(0);
                        if (atVal.equals("EXPN")) {

                            String expn = xmlStreamReader.getElementText();
                            ArrayList flist = expnHm.get(expn);
                            if (flist == null) {
//                                System.out.println("Tu khong co la: ");
//                                System.out.println(expn);
                                xmlStreamWriter.writeStartElement(localName);
                                xmlStreamWriter.writeAttribute("NSW", "SEQN");
                                xmlStreamWriter.writeCharacters(expn);
                                xmlStreamWriter.writeEndElement();

                            } else {
                                expnDoc.add(expn);
                                xmlStreamWriter.writeStartElement(localName);
                                xmlStreamWriter.writeAttribute("NSW", "EXPN");
                                xmlStreamWriter.writeCharacters(expn);
                                xmlStreamWriter.writeEndElement();
                            }
                        } else {
                            xmlStreamWriter.writeStartElement(localName);
                            xmlStreamWriter.writeAttribute("NSW", atVal);
                            String nsw = xmlStreamReader.getElementText();
                            xmlStreamWriter.writeCharacters(nsw);
                            xmlStreamWriter.writeEndElement();
                        }


                    }

                    if (localName.equals("SPLIT")) {
                        xmlStreamWriter.writeStartElement(localName);
                    }


                    break;

                case XMLStreamConstants.END_ELEMENT:
                    String endLocalName = xmlStreamReader.getLocalName();
                    //System.out.println(endLocalName);
                    if (endLocalName.equals("record")) {
                        xmlStreamWriter.writeEndElement();
                        expnAr.add(expnDoc);

                    }
                    if (endLocalName.equals("Infomation")) {
                        xmlStreamWriter.writeEndElement();

                    }

                    if (endLocalName.equals("SPLIT")) {
                        xmlStreamWriter.writeEndElement();
                    }



                    break;
                case XMLStreamConstants.CHARACTERS:
                    String character = xmlStreamReader.getText();
                    xmlStreamWriter.writeCharacters(character);
                    break;

                default:
                    break;
            }
        }


        xmlStreamWriter.writeEndElement();

        xmlStreamReader.close();
        xmlStreamWriter.close();
        fis.close();
        fos.close();

    }

    public void readDataFile() throws UnsupportedEncodingException, FileNotFoundException, IOException {
        BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream("data\\expand\\expn_full.db"), "UTF8"));
//           BufferedReader br=new BufferedReader(new InputStreamReader(new FileInputStream("data\\lm\\lm.db"), "UTF8"));
        String line;
        while ((line = br.readLine()) != null) {
            StringTokenizer st = new StringTokenizer(line, "|");
            while (st.hasMoreTokens()) {
                String expn = st.nextToken();
                String full = st.nextToken();
                int count = Integer.parseInt(st.nextToken());

                if (expnHm.containsKey(expn)) {
                    ArrayList fullAr = (ArrayList) expnHm.get(expn);
                    fullAr.add(full);
                } else {
                    ArrayList fullAr = new ArrayList();
                    fullAr.add(full);
                    expnHm.put(expn, fullAr);
                }
            }
        }
        br.close();
    }

    public void guessFull() throws FileNotFoundException, UnsupportedEncodingException, IOException {
        lm.loadModel("data//lm//lm.db");
        //Tim tap ucv
        for (int j = 0; j < expnAr.size(); j++) {
            try{
            System.out.println("abc:"+j);
            ArrayList expnDoc=(ArrayList)expnAr.get(j);
            if( expnDoc.isEmpty()){
                gFullAr.add(new ArrayList());
                continue;
            }
            ArrayList<ArrayList> UCV = new ArrayList<ArrayList>();
            for (int i = 0; i < expnDoc.size(); i++) {
                String expn = (String) expnDoc.get(i);
                ArrayList flist = expnHm.get(expn);
                UCV.add(flist);
            }
            // Chi so cua ucv
            ArrayList maxIndex = new ArrayList();
            for (int i = 0; i < UCV.size(); i++) {
                maxIndex.add(UCV.get(i).size() - 1);
            }

            //ucv dau tiên
            ArrayList firstUcvIndex = new ArrayList();
            for (int i = 0; i < maxIndex.size(); i++) {
                firstUcvIndex.add(0);
            }
            //Tinh gia tri p(e/f)*pf cho ucv đầu tiên
            ArrayList firstUcv = convertIndexToUcv(firstUcvIndex, UCV);
            double maxEntropy = calculatePef(firstUcv,expnDoc) * calculatePf(firstUcv);
            ArrayList maxUcv = firstUcv;

            ArrayList currentUcvIndex = firstUcvIndex;
            while (haveNextUCV(currentUcvIndex, maxIndex)) {
                ArrayList nextUcvIndex = nextUCV(currentUcvIndex, maxIndex);
                ArrayList nextUcv = convertIndexToUcv(nextUcvIndex, UCV);
                double pef=calculatePef(nextUcv, expnDoc);
                double pf=calculatePf(nextUcv);
//                double entropy = calculatePef(nextUcv,expnDoc) * calculatePf(nextUcv);
                double entropy=pef*pf;
                if (entropy > maxEntropy) {
                    maxEntropy = entropy;
                    maxUcv = nextUcv;
                    System.out.println("entropyla:"+entropy);
                }
                currentUcvIndex = nextUcvIndex;
            }

            gFullAr.add(maxUcv);
            }catch(Exception e){
            }
        }

    }

    public double calculatePef(ArrayList ucv, ArrayList expnDoc) {
        double pef = 1;
        for (int i = 0; i < ucv.size(); i++) {
            double cal=peff.calculate((String) expnDoc.get(i), (String) ucv.get(i));
            pef *= cal;
        }
        return pef;
    }

    public double calculatePf(ArrayList ucv) {
        return lm.getEntropy(ucv);
    }

    public boolean haveNextUCV(ArrayList ucvIndex, ArrayList maxIndex) {
        for(int i=0;i<ucvIndex.size();i++){
            int index=(Integer)ucvIndex.get(i);
            int max=(Integer)maxIndex.get(i);
            if(index<max){
                return true;
            }
        }
        return false;
    }

    public ArrayList nextUCV(ArrayList<Integer> ucvIndex, ArrayList<Integer> maxIndex) {
        for (int i = ucvIndex.size() - 1; i >= 0; i--) {
            if (ucvIndex.get(i) < maxIndex.get(i)) {
                int index = ucvIndex.get(i);
                ucvIndex.set(i, index+1);
                for(int j=ucvIndex.size()-1;j>i;j--){
                    ucvIndex.set(j, 0);
                }
                for(int l=0;l<ucvIndex.size();l++){
                    System.out.print(ucvIndex.get(l)+"    ");
                    
                }
                System.out.println();
                return ucvIndex;
            }
        }
        return null;
    }

    public ArrayList convertIndexToUcv(ArrayList<Integer> ucvIndex, ArrayList<ArrayList> UCV) {
        ArrayList ucv = new ArrayList();
        for (int i = 0; i < ucvIndex.size(); i++) {
            ucv.add(UCV.get(i).get(ucvIndex.get(i)));
        }
        return ucv;
    }

    public void writeToXMLFile(String ouf) throws FileNotFoundException, XMLStreamException, UnsupportedEncodingException, IOException {
        try{
        FileInputStream fis = new FileInputStream("textNor\\preExpn.xml");
        XMLStreamReader xmlStreamReader = XMLInputFactory.newInstance().createXMLStreamReader(fis);

        FileOutputStream fos = new FileOutputStream(ouf);
        OutputStreamWriter osw = new OutputStreamWriter(fos, "UTF8");
        XMLStreamWriter xmlStreamWriter = XMLOutputFactory.newInstance().createXMLStreamWriter(osw);

        xmlStreamWriter.writeStartElement("root");

        int index = 0;
        int docIndex=0;
        ArrayList gFullDoc=null;
        while (xmlStreamReader.hasNext()) {
            int eventCode = xmlStreamReader.next();
            switch (eventCode) {

                case XMLStreamConstants.START_ELEMENT:
                    String localName = xmlStreamReader.getLocalName();
                    if (localName.equals("record")) {
                        String RecId = xmlStreamReader.getAttributeValue(0);
                        xmlStreamWriter.writeStartElement(localName);
                        xmlStreamWriter.writeAttribute("RecId", RecId);
                        gFullDoc=(ArrayList)gFullAr.get(docIndex);
                        docIndex++;
                        System.out.println("doc: "+docIndex);

                    }

                    if (localName.equals("URL")) {
                        String url = xmlStreamReader.getElementText();

                        xmlStreamWriter.writeStartElement(localName);
                        xmlStreamWriter.writeCharacters(url);
                        xmlStreamWriter.writeEndElement();

                    }
                    if (localName.equals("Infomation")) {
                        xmlStreamWriter.writeStartElement(localName);


                    }

                    if (localName.equals("W")) {

                        String atVal = xmlStreamReader.getAttributeValue(0);
                        if (atVal.equals("EXPN")) {
                            String nsw = xmlStreamReader.getElementText();

                            xmlStreamWriter.writeStartElement(localName);
                            xmlStreamWriter.writeAttribute("NSW", "EXPN");
                            xmlStreamWriter.writeAttribute("FULL", (String) gFullDoc.get(index));
                            index++;
                            xmlStreamWriter.writeCharacters(nsw);
                            xmlStreamWriter.writeEndElement();

                        } else {
                            xmlStreamWriter.writeStartElement(localName);
                            xmlStreamWriter.writeAttribute("NSW", atVal);
                            String nsw = xmlStreamReader.getElementText();
                            xmlStreamWriter.writeCharacters(nsw);
                            xmlStreamWriter.writeEndElement();
                        }


                    }

                    if (localName.equals("SPLIT")) {
                        xmlStreamWriter.writeStartElement(localName);
                    }


                    break;

                case XMLStreamConstants.END_ELEMENT:
                    String endLocalName = xmlStreamReader.getLocalName();
                    //System.out.println(endLocalName);
                    if (endLocalName.equals("record")) {
                        xmlStreamWriter.writeEndElement();
                        index=0;
                    }
                    if (endLocalName.equals("Infomation")) {
                        xmlStreamWriter.writeEndElement();

                    }

                    if (endLocalName.equals("SPLIT")) {
                        xmlStreamWriter.writeEndElement();
                    }



                    break;
                case XMLStreamConstants.CHARACTERS:
                    String character = xmlStreamReader.getText();
                    xmlStreamWriter.writeCharacters(character);
                    break;

                default:
                    break;
            }
        }


        xmlStreamWriter.writeEndElement();

        xmlStreamReader.close();
        xmlStreamWriter.close();
        fis.close();
        fos.close();
       

        }
         catch(Exception e){
        }
    }

}
